{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo - Performance Comparison (CIFAR10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Load CIAFR10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "# https://github.com/RobustBench/robustbench\n",
    "from robustbench.data import load_cifar10\n",
    "from robustbench.utils import load_model, clean_accuracy\n",
    "from utils import l2_distance\n",
    "\n",
    "images, labels = load_cifar10(n_examples=10)\n",
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Standard Accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Standard\n",
      "- Standard Acc: 1.0\n",
      "Model: Wong2020Fast\n",
      "- Standard Acc: 1.0\n",
      "Model: Rice2020Overfitting\n",
      "- Standard Acc: 0.8\n"
     ]
    }
   ],
   "source": [
    "model_list = ['Standard', 'Wong2020Fast', 'Rice2020Overfitting']\n",
    "for model_name in model_list:\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    acc = clean_accuracy(model, images.to(device), labels.to(device))\n",
    "    print('Model: {}'.format(model_name))\n",
    "    print('- Standard Acc: {}'.format(acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Comparison with Foolbox and ART"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "foolbox 3.3.1\n",
      "art 1.7.0\n",
      "torchattacks 3.5.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-3-363377dfba62>:18: DeprecationWarning: The module art.classifiers will be removed in ART 1.8.0 and replaced with art.estimators.classification\n",
      "  from art.classifiers import PyTorchClassifier\n"
     ]
    }
   ],
   "source": [
    "import datetime\n",
    "import numpy as np\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(action='ignore')\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "# https://github.com/bethgelab/foolbox\n",
    "import foolbox as fb\n",
    "print(\"foolbox %s\"%(fb.__version__))\n",
    "\n",
    "# https://github.com/IBM/adversarial-robustness-toolbox\n",
    "import art\n",
    "import art.attacks.evasion as evasion\n",
    "from art.classifiers import PyTorchClassifier\n",
    "print(\"art %s\"%(art.__version__))\n",
    "\n",
    "import sys\n",
    "sys.path.insert(0, '..')\n",
    "# https://github.com/Harry24k/adversarial-attacks-pytorch\n",
    "import torchattacks\n",
    "print(\"torchattacks %s\"%(torchattacks.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.1. Linf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FGSM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\robustbench\\utils.py:109: DeprecationWarning: `norm` has been deprecated and will be removed in a future version.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Standard\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.4 (21 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.4 (10 ms)\n",
      "- ART\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\fast_gradient.py:362: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.num_random_init, (int, np.int)):\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\fast_gradient.py:502: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if x.dtype == np.object:\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\fast_gradient.py:412: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if grad.dtype != np.object and np.isnan(grad).any():\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\fast_gradient.py:446: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if batch.dtype == np.object:\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\fast_gradient.py:427: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if (grad.dtype != np.object and np.isinf(grad).any()) or np.isnan(grad.astype(np.float32)).any():\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\fast_gradient.py:462: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if perturbation_step.dtype != np.object:\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\fast_gradient.py:543: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if x_adv.dtype == np.object:\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\estimators\\pytorch.py:167: DeprecationWarning: `np.object` is a deprecated alias for the builtin `object`. To silence this warning, use `object` by itself. Doing this will not modify any behavior and is safe. \n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if self.all_framework_preprocessing and not (not input_is_tensor and x.dtype == np.object):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.4 (80 ms)\n",
      "\n",
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.6 (7 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.6 (6 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.6 (19 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.6 (21 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.6 (37 ms)\n",
      "- ART\n",
      "- Robust Acc: 0.6 (269 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.FGSM(model, eps=8/255)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.LinfFastGradientAttack(random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=8/255)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.FastGradientMethod(norm=np.inf, batch_size=50,\n",
    "                                     estimator=classifier, eps=8/255)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BIM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Standard\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 (101 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.0 (116 ms)\n",
      "- ART\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\projected_gradient_descent\\projected_gradient_descent.py:235: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.num_random_init, (int, np.int)):\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\projected_gradient_descent\\projected_gradient_descent_numpy.py:207: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.num_random_init, (int, np.int)):\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.0 (455 ms)\n",
      "\n",
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.4 (65 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.4 (65 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.4 (136 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.6 (486 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.6 (651 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.6 (1747 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.BIM(model, eps=8/255, alpha=2/255, steps=10)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.LinfBasicIterativeAttack(abs_stepsize=2/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=8/255)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.BasicIterativeMethod(batch_size=50,\n",
    "                                       estimator=classifier, eps=8/255,\n",
    "                                       eps_step=2/255, max_iter=10)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Standard\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 (59 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.0 (111 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.0 (447 ms)\n",
      "\n",
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.4 (61 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.4 (64 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.4 (146 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.6 (490 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.6 (656 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.6 (1713 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.PGD(model, eps=8/255, alpha=2/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.LinfPGD(abs_stepsize=2/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=8/255)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.ProjectedGradientDescent(batch_size=50, num_random_init=0,\n",
    "                                           estimator=classifier, eps=8/255,\n",
    "                                           eps_step=2/255, max_iter=10)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.2. L2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DeepFool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Standard\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 / L2: 0.16 (2609 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.0 / L2: 0.16 (1847 ms)\n",
      "- ART\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\deepfool.py:221: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.max_iter, (int, np.int)) or self.max_iter <= 0:\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\deepfool.py:224: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.nb_grads, (int, np.int)) or self.nb_grads <= 0:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff3b8366e13b4e90b5bef2e9fb4cd9c9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "DeepFool:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.2 / L2: 0.19 (14928 ms)\n",
      "\n",
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 / L2: 0.79 (695 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.4 / L2: 0.46 (306 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4612c009e8943819a351b1c138c7562",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "DeepFool:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.0 / L2: 0.78 (333 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 / L2: 0.61 (5979 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.1 / L2: 0.53 (5625 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2b94d0548ab4426b3b22e5fcf6e2e15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "DeepFool:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.0 / L2: 0.66 (5460 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.DeepFool(model, steps=50, overshoot=0.02)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "\n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.L2DeepFoolAttack(steps=50, candidates=10, overshoot=0.02)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda'), labels.to('cuda'), epsilons=1)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "\n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.DeepFool(classifier=classifier, max_iter=50,\n",
    "                           batch_size=50)\n",
    "\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### CW"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Standard\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 / L2: 0.34 (814 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.0 / L2: 0.31 (844 ms)\n",
      "- ART\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\carlini.py:491: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.binary_search_steps, (int, np.int)) or self.binary_search_steps < 0:\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\carlini.py:494: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.max_iter, (int, np.int)) or self.max_iter < 0:\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\carlini.py:497: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.max_halving, (int, np.int)) or self.max_halving < 1:\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\carlini.py:500: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.max_doubling, (int, np.int)) or self.max_doubling < 1:\n",
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\art\\attacks\\evasion\\carlini.py:503: DeprecationWarning: `np.int` is a deprecated alias for the builtin `int`. To silence this warning, use `int` by itself. Doing this will not modify any behavior and is safe. When replacing `np.int`, you may wish to use e.g. `np.int64` or `np.int32` to specify the precision. If you wish to review your current use, check the release note link for additional information.\n",
      "Deprecated in NumPy 1.20; for more details and guidance: https://numpy.org/devdocs/release/1.20.0-notes.html#deprecations\n",
      "  if not isinstance(self.batch_size, (int, np.int)) or self.batch_size < 1:\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc8c49d965774f2fa7189bebc9b548d4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "C&W L_2:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.0 / L2: 0.41 (37716 ms)\n",
      "\n",
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 / L2: 0.57 (995 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.1 / L2: 0.5 (1138 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "570b7e685587455597f5822c4a7d2800",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "C&W L_2:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.2 / L2: 0.71 (11814 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.2 / L2: 0.49 (14946 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.2 / L2: 0.49 (15185 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b00bb0fb3f4431e803830e48ac4410d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "C&W L_2:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.2 / L2: 0.57 (132464 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.CW(model, c=1, kappa=0, steps=100, lr=0.01)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.L2CarliniWagnerAttack(binary_search_steps=1, initial_const=1,\n",
    "                                           confidence=0, steps=100, stepsize=0.01)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda'), labels.to('cuda'), epsilons=1)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "\n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.CarliniL2Method(batch_size=50, classifier=classifier, \n",
    "                                  binary_search_steps=1, initial_const=1,\n",
    "                                  confidence=0, max_iter=100,\n",
    "                                  learning_rate=0.01)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PGD L2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Standard\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 / L2: 0.42 (60 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.0 / L2: 0.42 (120 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.0 / L2: 0.42 (454 ms)\n",
      "\n",
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.7 / L2: 0.5 (45 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.7 / L2: 0.5 (57 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.7 / L2: 0.5 (139 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.7 / L2: 0.5 (463 ms)\n",
      "- Foolbox\n",
      "- Robust Acc: 0.7 / L2: 0.5 (670 ms)\n",
      "- ART\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "PGD - Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "- Robust Acc: 0.7 / L2: 0.5 (1673 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').cuda()\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.PGDL2(model, eps=128/255, alpha=15/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Foolbox\")\n",
    "    fmodel = fb.PyTorchModel(model, bounds=(0, 1))\n",
    "    atk = fb.attacks.L2PGD(abs_stepsize=15/255, steps=10, random_start=False)\n",
    "    start = datetime.datetime.now()\n",
    "    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=128/255)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- ART\")\n",
    "    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),\n",
    "                                   loss=nn.CrossEntropyLoss(),\n",
    "                                   optimizer=optim.Adam(model.parameters(), lr=0.01),\n",
    "                                   input_shape=(3, 32, 32), nb_classes=10)\n",
    "    atk = evasion.ProjectedGradientDescent(batch_size=50, num_random_init=0,\n",
    "                                           norm = 2, estimator=classifier, eps=128/255,\n",
    "                                           eps_step=15/255, max_iter=10)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = torch.tensor(atk.generate(images, labels)).to(device)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    l2 = l2_distance(model, images, adv_images, labels, device=device)\n",
    "    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,\n",
    "                                                         int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Comparison with AutoAttack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\slcf\\anaconda3\\lib\\site-packages\\ipykernel\\ipkernel.py:287: DeprecationWarning: `should_run_async` will not call `transform_cell` automatically in the future. Please pass the result to `transformed_cell` argument and any exception that happen during thetransform in `preprocessing_exc_tuple` in IPython 7.17 and above.\n",
      "  and should_run_async(code)\n"
     ]
    }
   ],
   "source": [
    "# https://github.com/fra31/auto-attack\n",
    "from autoattack import AutoAttack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: Standard\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.0 (497 ms)\n",
      "- Torchattacks\n",
      "setting parameters for standard version\n",
      "using standard version including apgd-ce, apgd-t, fab-t, square\n",
      "initial accuracy: 100.00%\n",
      "apgd-ce - 1/1 - 10 out of 10 successfully perturbed\n",
      "robust accuracy after APGD-CE: 0.00% (total time 3.9 s)\n",
      "max Linf perturbation: 0.03137, nan in tensor: 0, max: 1.00000, min: 0.00000\n",
      "robust accuracy: 0.00%\n",
      "- Robust Acc: 0.0 (3888 ms)\n",
      "\n",
      "Model: Wong2020Fast\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.3 (24015 ms)\n",
      "- Torchattacks\n",
      "setting parameters for standard version\n",
      "using standard version including apgd-ce, apgd-t, fab-t, square\n",
      "initial accuracy: 100.00%\n",
      "apgd-ce - 1/1 - 6 out of 10 successfully perturbed\n",
      "robust accuracy after APGD-CE: 40.00% (total time 1.0 s)\n",
      "apgd-t - 1/1 - 1 out of 4 successfully perturbed\n",
      "robust accuracy after APGD-T: 30.00% (total time 8.7 s)\n",
      "fab-t - 1/1 - 0 out of 3 successfully perturbed\n",
      "robust accuracy after FAB-T: 30.00% (total time 23.3 s)\n",
      "square - 1/1 - 0 out of 3 successfully perturbed\n",
      "robust accuracy after SQUARE: 30.00% (total time 44.1 s)\n",
      "max Linf perturbation: 0.03137, nan in tensor: 0, max: 1.00000, min: 0.00000\n",
      "robust accuracy: 30.00%\n",
      "- Robust Acc: 0.3 (44104 ms)\n",
      "\n",
      "Model: Rice2020Overfitting\n",
      "- Torchattacks\n",
      "- Robust Acc: 0.3 (145918 ms)\n",
      "- Torchattacks\n",
      "setting parameters for standard version\n",
      "using standard version including apgd-ce, apgd-t, fab-t, square\n",
      "initial accuracy: 80.00%\n",
      "apgd-ce - 1/1 - 3 out of 8 successfully perturbed\n",
      "robust accuracy after APGD-CE: 50.00% (total time 13.4 s)\n",
      "apgd-t - 1/1 - 2 out of 5 successfully perturbed\n",
      "robust accuracy after APGD-T: 30.00% (total time 86.9 s)\n",
      "fab-t - 1/1 - 0 out of 3 successfully perturbed\n",
      "robust accuracy after FAB-T: 30.00% (total time 186.3 s)\n",
      "square - 1/1 - 0 out of 3 successfully perturbed\n",
      "robust accuracy after SQUARE: 30.00% (total time 312.2 s)\n",
      "max Linf perturbation: 0.03137, nan in tensor: 0, max: 1.00000, min: 0.00000\n",
      "robust accuracy: 30.00%\n",
      "- Robust Acc: 0.3 (312250 ms)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for model_name in model_list:\n",
    "    print('Model: {}'.format(model_name))\n",
    "    model = load_model(model_name, norm='Linf').to(device)\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = torchattacks.AutoAttack(model, eps=8/255)\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk(images, labels)\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print(\"- Torchattacks\")\n",
    "    atk = AutoAttack(model, norm='Linf', eps=8/255, version='standard')\n",
    "    start = datetime.datetime.now()\n",
    "    adv_images = atk.run_standard_evaluation(images, labels, bs=len(images)).cuda()\n",
    "    end = datetime.datetime.now()\n",
    "    acc = clean_accuracy(model, adv_images, labels)\n",
    "    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))\n",
    "    \n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
